#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :临时运行代码.py
# @Time      :2024/5/11 
# @Author    :CL
# @email     :1037654919@qq.com
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :ximalaya_video_cl.py
# @Time      :2024/4/19
# @Author    :CL
# @email     :1037654919@qq.com

import json
import os.path
import sys
import time
import pandas as pd
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
from config import settings as env_setting
from config.settings import INTERVAL
from utils.common_utils import logger, create_chrome_driver_linux
from selenium.webdriver.common.by import By
from util import mongo_manager
ximalaya_sound = mongo_manager('ximalaya_sound',db = 'public_data')

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Linux\""
}
def get_m4a_via_url(sound_url, driver):
    driver.get(sound_url)
    time.sleep(2 * INTERVAL)
    m4a_url = ''
    try:
        # 点击播放，才会触发下载动作
        play_ele_list = driver.find_elements(By.XPATH, '//*[@class="icon play-icon U_s"]')
        if len(play_ele_list) > 0:
            play_ele_list[0].click()
            time.sleep(3 * INTERVAL)
            for request in driver.requests:
                if "m4a" in request.url:
                    m4a_url =request.url
        else:
            logger.info(f"{sound_url}, no play button")
        return m4a_url
    except Exception as err:
        logger.exception(err)
        return m4a_url

def download_file_via_m4a_url(url, path,name):
    headers = {
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
    resp = requests.get(url, headers=headers)
    file_path = os.path.join(path,name+ '.'+url.split("/")[-1].split('.')[-1])
    if resp.status_code == 200:
        with open(file_path, "wb") as f:
            f.write(resp.content)

def down_sound(run_num = 1):
    try:
        logger.info(f"this is {sys.platform} operation system!!")
        # 创建模拟浏览器
        driver = create_chrome_driver_linux("")
        logger.info(f"create driver success！！{run_num}")
        # 抓取所有音频链接
    except:
        logger.exception(f"create_chrome_driver_linux 失败")
        return
    if driver:
        driver.get("https://www.ximalaya.com")
        time.sleep(2 * INTERVAL)
        with open(env_setting.LOGIN_COOKIE_PATH, 'r', encoding='utf-8') as fp:
            listCookies = json.load(fp)
        for cookie in listCookies:
            driver.add_cookie(cookie)
        driver.refresh()
        time.sleep(2* INTERVAL)
        while True:
            ele = ximalaya_sound.find_one_and_update({'_type':'普通话','album_url':'https://www.ximalaya.com/album/46466082','status':None},{'status':'running'}) #'_type':'百科'

            if ele:
                sound_url = ele["href"]
                name = str(ele["title"]).replace("/", "_").replace(":", "_").replace("：", "_").replace("|", "_").replace("|", "_")  #防止title里边有/导致保存出错
                h2 = str(ele["h2"]).replace("/", "_").replace(":", "_").replace("：", "_").replace("|", "_").replace("|", "_") #防止文件夹异常字符导致保存出错
                _type= ele["_type"]
                # 通过sound_url获取m4a的下载链接
                m4a_url = get_m4a_via_url(sound_url, driver=driver)
                if m4a_url.endswith(".m4a"):
                    ximalaya_sound.updateOne({'_id':ele['_id']}, {"m4a_url": m4a_url, "status": 1})
                else:
                    ximalaya_sound.updateOne({'_id':ele['_id']}, {"status": 'failed'})
                    continue
                #    下载 可以单独下载，也可以插于此处，刚好下载的时间就是chromedriver休息的时间
                path = os.path.join("/home/chenglei3/work/data/ximalaya/",_type, h2) #指定下载目录
                os.makedirs(path,exist_ok=True)
                #
                try:
                    download_file_via_m4a_url(str(m4a_url), path=path,name=name)
                    print(f"{name} m4a download success")

                    # 下载文本
                    if ele['album_url'] in  ['https://www.ximalaya.com/album/5822114',
                             'https://www.ximalaya.com/album/33004172',
                             # 'https://www.ximalaya.com/album/46595930',
                             'https://www.ximalaya.com/album/46466082',
                                 'https://www.ximalaya.com/album/50552092',
                                 'https://www.ximalaya.com/album/21328128',
                                 'https://www.ximalaya.com/album/2808888']:
                        try:
                            res = requests.get(sound_url, headers=headers, timeout=10)
                            soup = BeautifulSoup(res.text, 'lxml')
                            # 获取文本（声音简介）
                            article = soup.find('article', class_='intro').text
                            with open(f'{path}/{name}.txt', 'a', encoding='utf-8') as f:
                                f.write(article)
                            ximalaya_sound.updateOne({'_id': ele['_id']}, {"text": article})
                        except:
                            logger.exception(f"{name} 获取文本失败")
                    # 更新数据库
                    ximalaya_sound.updateOne({'_id':ele['_id']}, {"m4a_url": m4a_url,"status": 'done'})
                except  Exception as err:
                    logger.exception(err)
                    ximalaya_sound.updateOne({'_id': ele['_id']}, {"m4a_url": m4a_url, "status": 'failed'})
            else:
                break


def main():
    pool = Pool(processes=2)
    pool.map(down_sound, range(1,3))
    pool.close()
    pool.join()
    pass

if __name__ == '__main__':

    # ximalaya_sound.updateMany({}, {"status": None})
    # ximalaya_sound.updateMany({'m4a_url':''},{'status':None})
    # main() # 多线程
    down_sound() # 单线程
    ximalaya_sound.close()
